# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html

import json
import codecs

import MySQLdb
import MySQLdb.cursors

from twisted.enterprise import adbapi
import mysql_tools

class JsonPipeline(object):
    def __init__(self):  # 添加一下初始化方法
        self.ids_seen = set()
        self.file = codecs.open('index.json', 'wb', encoding='utf-8')

    def open_spider(self, spider):
        if spider.name == 'vxinghe':
            self.file = codecs.open('vxinghe.json', 'wb', encoding='utf-8')
        self.file.write('[')

    def close_spider(self, spider):
        self.file.write(']')

    def process_item(self, item, spider):
        # 去重
        # if item['id'] not in self.ids_seen:
        #     self.ids_seen.add(item['id'])
        # 这一句会将你每次返回的字典抓取出来,“ensure_ascii=False”这一句话很重要，如果是True的话就是我们保存的\u4e2d\u56fd这种格式了
        line = json.dumps(dict(item), ensure_ascii=False) + ',\n'
        self.file.write(line)  # 写入到文件中
        return item

class MysqlPipeline(object):
    def __init__(self, dbpool):
        self.dbpool = dbpool

    def open_spider(self, spider):
        pass

    def close_spider(self, spider):
        mysql_tools.document_save_to_file()

    @classmethod
    def from_settings(cls, settings):
        '''1、@classmethod声明一个类方法，而对于平常我们见到的则叫做实例方法。
           2、类方法的第一个参数cls（class的缩写，指这个类本身），而实例方法的第一个参数是self，表示该类的一个实例
           3、可以通过类来调用，就像C.f()，相当于java中的静态方法'''
        dbparams = dict(
            host=settings['MYSQL_HOST'],  # 读取settings中的配置
            db=settings['MYSQL_DBNAME'],
            user=settings['MYSQL_USER'],
            passwd=settings['MYSQL_PASSWD'],
            charset='utf8',  # 编码要加上，否则可能出现中文乱码问题
            cursorclass=MySQLdb.cursors.DictCursor,
            use_unicode=True,
        )
        dbpool = adbapi.ConnectionPool('MySQLdb', **dbparams)  # **表示将字典扩展为关键字参数,相当于host=xxx,db=yyy....
        return cls(dbpool)  # 相当于dbpool付给了这个类，self中可以得到

    # 错误处理方法
    def _handle_error(self, failue, item, spider):
        print('出错啦。。。。。。。。。。', failue)

    # pipeline默认调用
    def process_item(self, item, spider):
        if spider.name == 'sina':
            query = self.dbpool.runInteraction(self.insert_to_news_guonei, item)
            query.addErrback(self._handle_error, item, spider)
        elif spider.name == 'guonei':
            query = self.dbpool.runInteraction(self.insert_to_news_guonei, item)
            query.addErrback(self._handle_error, item, spider)
        return item

    # 向zreading表中添加数据
    def insert_to_news_guonei(self, tx, item):
        print('----------------head-------------------')
        if item.has_key('content'):
            print('---------update news to guonei-----------', type(item["title"]))
            sql = "update document2 set content='" + item["content"] + "' where title='" + item['title'] + "'"
            tx.execute(sql)
        else:
            print('---------add news to guonei-----------', type(item["title"]))
            print(item["name"])
            print('----------------foot-------------------')
            sql = "insert into document2(title, category_id, source, create_time) values(%s, %s, %s, %s)"
            params = (item["title"], item["category_id"], item["source"], item["create_time"])
            tx.execute(sql, params)
            sql = "update document2 set name='" + item["name"] + "' where title='" + item['title'] + "'"
            tx.execute(sql)
            if item.has_key('cover'):
                sql = "update document2 set cover='" + item["cover"] + "' where title='" + item['title'] + "'"
                tx.execute(sql)


    # def insert_to_news_guoji(self, tx, item):
    #     print('---------add news to news_guoji-----------')
    #     print('----------------head-------------------')
    #     print(item)
    #     print('----------------foot-------------------')
    #     sql = "insert into document2(title, category_id, source, create_time, description, content) values(%s, %s, %s, %s, %s, %s)"
    #     params = (item["title"], item["category_id"], item["source"], item["create_time"], item["description"], item["content"])
        # tx.execute(sql, params)

        # sql = "select * from document"
        # res = tx.execute(sql)
        # results = tx.fetchall()
        # for row in results:
            # print('OOOOOOOOOOO', row[1], row['user'])
            # if row['user'] == 'root':
            #     print("result======aaa=======", row)

